{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import tabulate\n",
    "import pprint\n",
    "import click\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from ray.tune.commands import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load data and general exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "browser = RayTuneExperimentBrowser(os.path.expanduser(\"~/nta/results/VGG19SparseFull\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/lsouza/miniconda3/envs/numenta/lib/python3.7/site-packages/numpy/core/fromnumeric.py:3118: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/Users/lsouza/miniconda3/envs/numenta/lib/python3.7/site-packages/numpy/core/_methods.py:85: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "df = browser.best_experiments(min_test_accuracy=0.0, min_noise_accuracy=0.0, sort_by=\"test_accuracy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Experiment Name</th>\n",
       "      <th>test_accuracy</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>epoch_test_accuracy</th>\n",
       "      <th>noise_accuracy</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>epoch_noise_accuracy</th>\n",
       "      <th>epochs</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <th>...</th>\n",
       "      <th>path</th>\n",
       "      <th>repetitions</th>\n",
       "      <th>restore_supported</th>\n",
       "      <th>sync_function</th>\n",
       "      <th>test_batch_size</th>\n",
       "      <th>test_batches_in_epoch</th>\n",
       "      <th>upload_dir</th>\n",
       "      <th>use_max_pooling</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>weight_sparsity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9_batch_size=128,batches_in_epoch=403,boost_st...</td>\n",
       "      <td>0.7021</td>\n",
       "      <td>0.7021</td>\n",
       "      <td>89</td>\n",
       "      <td>0.2357</td>\n",
       "      <td>0.2584</td>\n",
       "      <td>52</td>\n",
       "      <td>90</td>\n",
       "      <td>128</td>\n",
       "      <td>403</td>\n",
       "      <td>...</td>\n",
       "      <td>~/nta/results</td>\n",
       "      <td>150</td>\n",
       "      <td>True</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000990</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>175_batch_size=128,batches_in_epoch=425,boost_...</td>\n",
       "      <td>0.7017</td>\n",
       "      <td>0.7091</td>\n",
       "      <td>140</td>\n",
       "      <td>0.2208</td>\n",
       "      <td>0.2528</td>\n",
       "      <td>87</td>\n",
       "      <td>164</td>\n",
       "      <td>128</td>\n",
       "      <td>425</td>\n",
       "      <td>...</td>\n",
       "      <td>~/nta/results</td>\n",
       "      <td>150</td>\n",
       "      <td>True</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000609</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>183_batch_size=128,batches_in_epoch=529,boost_...</td>\n",
       "      <td>0.7005</td>\n",
       "      <td>0.7005</td>\n",
       "      <td>89</td>\n",
       "      <td>0.2537</td>\n",
       "      <td>0.2821</td>\n",
       "      <td>29</td>\n",
       "      <td>90</td>\n",
       "      <td>128</td>\n",
       "      <td>529</td>\n",
       "      <td>...</td>\n",
       "      <td>~/nta/results</td>\n",
       "      <td>150</td>\n",
       "      <td>True</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000832</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>277</th>\n",
       "      <td>277_batch_size=128,batches_in_epoch=518,boost_...</td>\n",
       "      <td>0.6990</td>\n",
       "      <td>0.6990</td>\n",
       "      <td>89</td>\n",
       "      <td>0.2584</td>\n",
       "      <td>0.2932</td>\n",
       "      <td>71</td>\n",
       "      <td>90</td>\n",
       "      <td>128</td>\n",
       "      <td>518</td>\n",
       "      <td>...</td>\n",
       "      <td>~/nta/results</td>\n",
       "      <td>150</td>\n",
       "      <td>True</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000680</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11_batch_size=128,batches_in_epoch=564,boost_s...</td>\n",
       "      <td>0.6965</td>\n",
       "      <td>0.6989</td>\n",
       "      <td>159</td>\n",
       "      <td>0.2341</td>\n",
       "      <td>0.2620</td>\n",
       "      <td>85</td>\n",
       "      <td>164</td>\n",
       "      <td>128</td>\n",
       "      <td>564</td>\n",
       "      <td>...</td>\n",
       "      <td>~/nta/results</td>\n",
       "      <td>150</td>\n",
       "      <td>True</td>\n",
       "      <td>aws s3 sync `dirname {local_dir}` {remote_dir}...</td>\n",
       "      <td>128</td>\n",
       "      <td>500</td>\n",
       "      <td>s3://lsouza/ray/results</td>\n",
       "      <td>True</td>\n",
       "      <td>0.000698</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 49 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       Experiment Name  test_accuracy  \\\n",
       "9    9_batch_size=128,batches_in_epoch=403,boost_st...         0.7021   \n",
       "175  175_batch_size=128,batches_in_epoch=425,boost_...         0.7017   \n",
       "183  183_batch_size=128,batches_in_epoch=529,boost_...         0.7005   \n",
       "277  277_batch_size=128,batches_in_epoch=518,boost_...         0.6990   \n",
       "11   11_batch_size=128,batches_in_epoch=564,boost_s...         0.6965   \n",
       "\n",
       "     test_accuracy_max epoch_test_accuracy  noise_accuracy  \\\n",
       "9               0.7021                  89          0.2357   \n",
       "175             0.7091                 140          0.2208   \n",
       "183             0.7005                  89          0.2537   \n",
       "277             0.6990                  89          0.2584   \n",
       "11              0.6989                 159          0.2341   \n",
       "\n",
       "     noise_accuracy_max epoch_noise_accuracy epochs batch_size  \\\n",
       "9                0.2584                   52     90        128   \n",
       "175              0.2528                   87    164        128   \n",
       "183              0.2821                   29     90        128   \n",
       "277              0.2932                   71     90        128   \n",
       "11               0.2620                   85    164        128   \n",
       "\n",
       "    batches_in_epoch  ...           path  repetitions  restore_supported  \\\n",
       "9                403  ...  ~/nta/results          150               True   \n",
       "175              425  ...  ~/nta/results          150               True   \n",
       "183              529  ...  ~/nta/results          150               True   \n",
       "277              518  ...  ~/nta/results          150               True   \n",
       "11               564  ...  ~/nta/results          150               True   \n",
       "\n",
       "                                         sync_function test_batch_size  \\\n",
       "9    aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "175  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "183  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "277  aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "11   aws s3 sync `dirname {local_dir}` {remote_dir}...             128   \n",
       "\n",
       "     test_batches_in_epoch               upload_dir  use_max_pooling  \\\n",
       "9                      500  s3://lsouza/ray/results             True   \n",
       "175                    500  s3://lsouza/ray/results             True   \n",
       "183                    500  s3://lsouza/ray/results             True   \n",
       "277                    500  s3://lsouza/ray/results             True   \n",
       "11                     500  s3://lsouza/ray/results             True   \n",
       "\n",
       "     weight_decay  weight_sparsity  \n",
       "9        0.000990              NaN  \n",
       "175      0.000609              NaN  \n",
       "183      0.000832              NaN  \n",
       "277      0.000680              NaN  \n",
       "11       0.000698              NaN  \n",
       "\n",
       "[5 rows x 49 columns]"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Experiment Name', 'test_accuracy', 'test_accuracy_max',\n",
       "       'epoch_test_accuracy', 'noise_accuracy', 'noise_accuracy_max',\n",
       "       'epoch_noise_accuracy', 'epochs', 'batch_size', 'batches_in_epoch',\n",
       "       'batches_in_first_epoch', 'block_sizes', 'boost_strength',\n",
       "       'boost_strength_factor', 'checkpoint_at_end', 'cnn_kernel_size',\n",
       "       'cnn_out_channels', 'cnn_percent_on', 'cnn_weight_sparsity',\n",
       "       'cpu_percentage', 'data_dir', 'dataset', 'experiment',\n",
       "       'first_epoch_batch_size', 'gpu_percentage', 'input_shape', 'iterations',\n",
       "       'k_inference_factor', 'learning_rate', 'learning_rate_gamma',\n",
       "       'linear_n', 'linear_percent_on', 'lr_step_schedule', 'momentum', 'name',\n",
       "       'network_type', 'num_cpus', 'num_gpus', 'output_size', 'path',\n",
       "       'repetitions', 'restore_supported', 'sync_function', 'test_batch_size',\n",
       "       'test_batches_in_epoch', 'upload_dir', 'use_max_pooling',\n",
       "       'weight_decay', 'weight_sparsity'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Experiment Name           9_batch_size=128,batches_in_epoch=403,boost_st...\n",
       "test_accuracy                                                        0.7021\n",
       "test_accuracy_max                                                    0.7021\n",
       "epoch_test_accuracy                                                      89\n",
       "noise_accuracy                                                       0.2357\n",
       "noise_accuracy_max                                                   0.2584\n",
       "epoch_noise_accuracy                                                     52\n",
       "epochs                                                                   90\n",
       "batch_size                                                              128\n",
       "batches_in_epoch                                                        403\n",
       "batches_in_first_epoch                                                  600\n",
       "block_sizes                                                             3.2\n",
       "boost_strength                                                      1.73176\n",
       "boost_strength_factor                                              0.626255\n",
       "checkpoint_at_end                                                      True\n",
       "cnn_kernel_size                                                           3\n",
       "cnn_out_channels                                                      294.4\n",
       "cnn_percent_on                                                      0.31395\n",
       "cnn_weight_sparsity                                                0.917248\n",
       "cpu_percentage                                                            1\n",
       "data_dir                                                     ~/nta/datasets\n",
       "dataset                                                            CIFAR100\n",
       "experiment                                                             grid\n",
       "first_epoch_batch_size                                                    4\n",
       "gpu_percentage                                                        0.165\n",
       "input_shape                                                         22.3333\n",
       "iterations                                                              164\n",
       "k_inference_factor                                                 0.977242\n",
       "learning_rate                                                      0.102157\n",
       "learning_rate_gamma                                               0.0510721\n",
       "linear_n                                                                NaN\n",
       "linear_percent_on                                                       NaN\n",
       "lr_step_schedule                                                      101.5\n",
       "momentum                                                           0.399231\n",
       "name                                                        VGG19SparseFull\n",
       "network_type                                                            vgg\n",
       "num_cpus                                                                 31\n",
       "num_gpus                                                                  4\n",
       "output_size                                                             100\n",
       "path                                                          ~/nta/results\n",
       "repetitions                                                             150\n",
       "restore_supported                                                      True\n",
       "sync_function             aws s3 sync `dirname {local_dir}` {remote_dir}...\n",
       "test_batch_size                                                         128\n",
       "test_batches_in_epoch                                                   500\n",
       "upload_dir                                          s3://lsouza/ray/results\n",
       "use_max_pooling                                                        True\n",
       "weight_decay                                                     0.00099037\n",
       "weight_sparsity                                                         NaN\n",
       "Name: 9, dtype: object"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.iloc[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Epochs and Accuracy exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['epochs']==164])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.278818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <td>-0.278818</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    test_accuracy_max  noise_accuracy_max\n",
       "test_accuracy_max            1.000000           -0.278818\n",
       "noise_accuracy_max          -0.278818            1.000000"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==164][['test_accuracy_max', 'noise_accuracy_max']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy_max     0.5826\n",
       "noise_accuracy_max    0.2199\n",
       "dtype: float64"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==164][['test_accuracy_max', 'noise_accuracy_max']].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy_max     0.663829\n",
       "noise_accuracy_max    0.288516\n",
       "dtype: float64"
      ]
     },
     "execution_count": 158,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==164][['test_accuracy_max', 'noise_accuracy_max']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy_max     0.7091\n",
       "noise_accuracy_max    0.3271\n",
       "dtype: float64"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==164][['test_accuracy_max', 'noise_accuracy_max']].max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['epochs']==90])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.584994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <td>0.584994</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    test_accuracy_max  noise_accuracy_max\n",
       "test_accuracy_max            1.000000            0.584994\n",
       "noise_accuracy_max           0.584994            1.000000"
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==90][['test_accuracy_max', 'noise_accuracy_max']].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy_max     0.4906\n",
       "noise_accuracy_max    0.1592\n",
       "dtype: float64"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==90][['test_accuracy_max', 'noise_accuracy_max']].min()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy_max     0.638398\n",
       "noise_accuracy_max    0.252256\n",
       "dtype: float64"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==90][['test_accuracy_max', 'noise_accuracy_max']].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "test_accuracy_max     0.7021\n",
       "noise_accuracy_max    0.2967\n",
       "dtype: float64"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']==90][['test_accuracy_max', 'noise_accuracy_max']].max()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- It is interesting to see that the experiments that run 90 epochs have a very different correlation between noise and test accuracy than the experiments that run 164 epochs, even though the averages are very similar. What can that mean? After some point progress in test accuracy can lead to regress in noise accuracy? Which would imply the more the model fits to the standard data, the lesser the noise accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>test_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>epochs</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.830726</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test_accuracy</th>\n",
       "      <td>0.830726</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 epochs  test_accuracy\n",
       "epochs         1.000000       0.830726\n",
       "test_accuracy  0.830726       1.000000"
      ]
     },
     "execution_count": 185,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']>=30][['epochs', 'test_accuracy']].astype(np.float32).corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epochs</th>\n",
       "      <th>noise_accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>epochs</th>\n",
       "      <td>1.00000</td>\n",
       "      <td>0.80023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noise_accuracy</th>\n",
       "      <td>0.80023</td>\n",
       "      <td>1.00000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 epochs  noise_accuracy\n",
       "epochs          1.00000         0.80023\n",
       "noise_accuracy  0.80023         1.00000"
      ]
     },
     "execution_count": 186,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[df['epochs']>=30][['epochs', 'noise_accuracy']].astype(np.float32).corr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Test accuracy seems more correlated to number of epochs then test accuracy, but the difference is small, might be due to randomness"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A look at other possible correlations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>momentum</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <th>boost_strength</th>\n",
       "      <th>boost_strength_factor</th>\n",
       "      <th>k_inference_factor</th>\n",
       "      <th>cnn_percent_on</th>\n",
       "      <th>cnn_weight_sparsity</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>learning_rate</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012162</td>\n",
       "      <td>-0.017390</td>\n",
       "      <td>0.057644</td>\n",
       "      <td>-0.037703</td>\n",
       "      <td>-0.015762</td>\n",
       "      <td>-0.117501</td>\n",
       "      <td>-0.002551</td>\n",
       "      <td>-0.034224</td>\n",
       "      <td>-0.017177</td>\n",
       "      <td>0.001650</td>\n",
       "      <td>-0.145329</td>\n",
       "      <td>-0.097126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <td>0.012162</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.013388</td>\n",
       "      <td>0.037070</td>\n",
       "      <td>-0.023368</td>\n",
       "      <td>-0.035762</td>\n",
       "      <td>0.029487</td>\n",
       "      <td>0.068997</td>\n",
       "      <td>-0.034483</td>\n",
       "      <td>-0.121417</td>\n",
       "      <td>0.063874</td>\n",
       "      <td>-0.039836</td>\n",
       "      <td>-0.033162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_decay</th>\n",
       "      <td>-0.017390</td>\n",
       "      <td>0.013388</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.038532</td>\n",
       "      <td>-0.015457</td>\n",
       "      <td>-0.064426</td>\n",
       "      <td>-0.061364</td>\n",
       "      <td>0.088593</td>\n",
       "      <td>0.048938</td>\n",
       "      <td>-0.037099</td>\n",
       "      <td>-0.074064</td>\n",
       "      <td>-0.085577</td>\n",
       "      <td>-0.047994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>momentum</th>\n",
       "      <td>0.057644</td>\n",
       "      <td>0.037070</td>\n",
       "      <td>0.038532</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.058849</td>\n",
       "      <td>0.058075</td>\n",
       "      <td>0.009580</td>\n",
       "      <td>-0.032437</td>\n",
       "      <td>-0.055015</td>\n",
       "      <td>0.007072</td>\n",
       "      <td>-0.106112</td>\n",
       "      <td>-0.383913</td>\n",
       "      <td>-0.276776</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>batch_size</th>\n",
       "      <td>-0.037703</td>\n",
       "      <td>-0.023368</td>\n",
       "      <td>-0.015457</td>\n",
       "      <td>0.058849</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040585</td>\n",
       "      <td>0.018939</td>\n",
       "      <td>-0.066548</td>\n",
       "      <td>-0.045015</td>\n",
       "      <td>0.007049</td>\n",
       "      <td>0.120368</td>\n",
       "      <td>0.136283</td>\n",
       "      <td>0.146644</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <td>-0.015762</td>\n",
       "      <td>-0.035762</td>\n",
       "      <td>-0.064426</td>\n",
       "      <td>0.058075</td>\n",
       "      <td>0.040585</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.012808</td>\n",
       "      <td>-0.055749</td>\n",
       "      <td>-0.001159</td>\n",
       "      <td>-0.042075</td>\n",
       "      <td>-0.062545</td>\n",
       "      <td>0.115584</td>\n",
       "      <td>0.107763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boost_strength</th>\n",
       "      <td>-0.117501</td>\n",
       "      <td>0.029487</td>\n",
       "      <td>-0.061364</td>\n",
       "      <td>0.009580</td>\n",
       "      <td>0.018939</td>\n",
       "      <td>0.012808</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.090654</td>\n",
       "      <td>0.077328</td>\n",
       "      <td>-0.026559</td>\n",
       "      <td>0.087035</td>\n",
       "      <td>0.040313</td>\n",
       "      <td>0.022680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boost_strength_factor</th>\n",
       "      <td>-0.002551</td>\n",
       "      <td>0.068997</td>\n",
       "      <td>0.088593</td>\n",
       "      <td>-0.032437</td>\n",
       "      <td>-0.066548</td>\n",
       "      <td>-0.055749</td>\n",
       "      <td>-0.090654</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.011040</td>\n",
       "      <td>-0.028418</td>\n",
       "      <td>-0.104310</td>\n",
       "      <td>0.009095</td>\n",
       "      <td>0.003375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_inference_factor</th>\n",
       "      <td>-0.034224</td>\n",
       "      <td>-0.034483</td>\n",
       "      <td>0.048938</td>\n",
       "      <td>-0.055015</td>\n",
       "      <td>-0.045015</td>\n",
       "      <td>-0.001159</td>\n",
       "      <td>0.077328</td>\n",
       "      <td>-0.011040</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003868</td>\n",
       "      <td>0.059874</td>\n",
       "      <td>-0.043341</td>\n",
       "      <td>-0.071874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn_percent_on</th>\n",
       "      <td>-0.017177</td>\n",
       "      <td>-0.121417</td>\n",
       "      <td>-0.037099</td>\n",
       "      <td>0.007072</td>\n",
       "      <td>0.007049</td>\n",
       "      <td>-0.042075</td>\n",
       "      <td>-0.026559</td>\n",
       "      <td>-0.028418</td>\n",
       "      <td>0.003868</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.004209</td>\n",
       "      <td>0.336341</td>\n",
       "      <td>0.368590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn_weight_sparsity</th>\n",
       "      <td>0.001650</td>\n",
       "      <td>0.063874</td>\n",
       "      <td>-0.074064</td>\n",
       "      <td>-0.106112</td>\n",
       "      <td>0.120368</td>\n",
       "      <td>-0.062545</td>\n",
       "      <td>0.087035</td>\n",
       "      <td>-0.104310</td>\n",
       "      <td>0.059874</td>\n",
       "      <td>-0.004209</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.061532</td>\n",
       "      <td>0.042322</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <td>-0.145329</td>\n",
       "      <td>-0.039836</td>\n",
       "      <td>-0.085577</td>\n",
       "      <td>-0.383913</td>\n",
       "      <td>0.136283</td>\n",
       "      <td>0.115584</td>\n",
       "      <td>0.040313</td>\n",
       "      <td>0.009095</td>\n",
       "      <td>-0.043341</td>\n",
       "      <td>0.336341</td>\n",
       "      <td>0.061532</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.952541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <td>-0.097126</td>\n",
       "      <td>-0.033162</td>\n",
       "      <td>-0.047994</td>\n",
       "      <td>-0.276776</td>\n",
       "      <td>0.146644</td>\n",
       "      <td>0.107763</td>\n",
       "      <td>0.022680</td>\n",
       "      <td>0.003375</td>\n",
       "      <td>-0.071874</td>\n",
       "      <td>0.368590</td>\n",
       "      <td>0.042322</td>\n",
       "      <td>0.952541</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       learning_rate  learning_rate_gamma  weight_decay  \\\n",
       "learning_rate               1.000000             0.012162     -0.017390   \n",
       "learning_rate_gamma         0.012162             1.000000      0.013388   \n",
       "weight_decay               -0.017390             0.013388      1.000000   \n",
       "momentum                    0.057644             0.037070      0.038532   \n",
       "batch_size                 -0.037703            -0.023368     -0.015457   \n",
       "batches_in_epoch           -0.015762            -0.035762     -0.064426   \n",
       "boost_strength             -0.117501             0.029487     -0.061364   \n",
       "boost_strength_factor      -0.002551             0.068997      0.088593   \n",
       "k_inference_factor         -0.034224            -0.034483      0.048938   \n",
       "cnn_percent_on             -0.017177            -0.121417     -0.037099   \n",
       "cnn_weight_sparsity         0.001650             0.063874     -0.074064   \n",
       "noise_accuracy_max         -0.145329            -0.039836     -0.085577   \n",
       "test_accuracy_max          -0.097126            -0.033162     -0.047994   \n",
       "\n",
       "                       momentum  batch_size  batches_in_epoch  boost_strength  \\\n",
       "learning_rate          0.057644   -0.037703         -0.015762       -0.117501   \n",
       "learning_rate_gamma    0.037070   -0.023368         -0.035762        0.029487   \n",
       "weight_decay           0.038532   -0.015457         -0.064426       -0.061364   \n",
       "momentum               1.000000    0.058849          0.058075        0.009580   \n",
       "batch_size             0.058849    1.000000          0.040585        0.018939   \n",
       "batches_in_epoch       0.058075    0.040585          1.000000        0.012808   \n",
       "boost_strength         0.009580    0.018939          0.012808        1.000000   \n",
       "boost_strength_factor -0.032437   -0.066548         -0.055749       -0.090654   \n",
       "k_inference_factor    -0.055015   -0.045015         -0.001159        0.077328   \n",
       "cnn_percent_on         0.007072    0.007049         -0.042075       -0.026559   \n",
       "cnn_weight_sparsity   -0.106112    0.120368         -0.062545        0.087035   \n",
       "noise_accuracy_max    -0.383913    0.136283          0.115584        0.040313   \n",
       "test_accuracy_max     -0.276776    0.146644          0.107763        0.022680   \n",
       "\n",
       "                       boost_strength_factor  k_inference_factor  \\\n",
       "learning_rate                      -0.002551           -0.034224   \n",
       "learning_rate_gamma                 0.068997           -0.034483   \n",
       "weight_decay                        0.088593            0.048938   \n",
       "momentum                           -0.032437           -0.055015   \n",
       "batch_size                         -0.066548           -0.045015   \n",
       "batches_in_epoch                   -0.055749           -0.001159   \n",
       "boost_strength                     -0.090654            0.077328   \n",
       "boost_strength_factor               1.000000           -0.011040   \n",
       "k_inference_factor                 -0.011040            1.000000   \n",
       "cnn_percent_on                     -0.028418            0.003868   \n",
       "cnn_weight_sparsity                -0.104310            0.059874   \n",
       "noise_accuracy_max                  0.009095           -0.043341   \n",
       "test_accuracy_max                   0.003375           -0.071874   \n",
       "\n",
       "                       cnn_percent_on  cnn_weight_sparsity  \\\n",
       "learning_rate               -0.017177             0.001650   \n",
       "learning_rate_gamma         -0.121417             0.063874   \n",
       "weight_decay                -0.037099            -0.074064   \n",
       "momentum                     0.007072            -0.106112   \n",
       "batch_size                   0.007049             0.120368   \n",
       "batches_in_epoch            -0.042075            -0.062545   \n",
       "boost_strength              -0.026559             0.087035   \n",
       "boost_strength_factor       -0.028418            -0.104310   \n",
       "k_inference_factor           0.003868             0.059874   \n",
       "cnn_percent_on               1.000000            -0.004209   \n",
       "cnn_weight_sparsity         -0.004209             1.000000   \n",
       "noise_accuracy_max           0.336341             0.061532   \n",
       "test_accuracy_max            0.368590             0.042322   \n",
       "\n",
       "                       noise_accuracy_max  test_accuracy_max  \n",
       "learning_rate                   -0.145329          -0.097126  \n",
       "learning_rate_gamma             -0.039836          -0.033162  \n",
       "weight_decay                    -0.085577          -0.047994  \n",
       "momentum                        -0.383913          -0.276776  \n",
       "batch_size                       0.136283           0.146644  \n",
       "batches_in_epoch                 0.115584           0.107763  \n",
       "boost_strength                   0.040313           0.022680  \n",
       "boost_strength_factor            0.009095           0.003375  \n",
       "k_inference_factor              -0.043341          -0.071874  \n",
       "cnn_percent_on                   0.336341           0.368590  \n",
       "cnn_weight_sparsity              0.061532           0.042322  \n",
       "noise_accuracy_max               1.000000           0.952541  \n",
       "test_accuracy_max                0.952541           1.000000  "
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tunable_params_general = ['learning_rate', 'learning_rate_gamma', 'weight_decay', 'momentum', 'batch_size', 'batches_in_epoch']\n",
    "tunable_params_sparsity = ['boost_strength', 'boost_strength_factor', 'k_inference_factor', 'cnn_percent_on', 'cnn_weight_sparsity']\n",
    "tunable_params = tunable_params_general + tunable_params_sparsity\n",
    "performance_metrics = ['noise_accuracy_max', 'test_accuracy_max']\n",
    "corr_params = tunable_params + performance_metrics\n",
    "\n",
    "df[corr_params].astype(np.float32).corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>momentum</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <th>boost_strength</th>\n",
       "      <th>boost_strength_factor</th>\n",
       "      <th>k_inference_factor</th>\n",
       "      <th>cnn_percent_on</th>\n",
       "      <th>cnn_weight_sparsity</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>learning_rate</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_decay</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>momentum</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>batch_size</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boost_strength</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boost_strength_factor</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_inference_factor</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn_percent_on</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn_weight_sparsity</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       learning_rate  learning_rate_gamma  weight_decay  \\\n",
       "learning_rate                   True                False         False   \n",
       "learning_rate_gamma            False                 True         False   \n",
       "weight_decay                   False                False          True   \n",
       "momentum                       False                False         False   \n",
       "batch_size                     False                False         False   \n",
       "batches_in_epoch               False                False         False   \n",
       "boost_strength                 False                False         False   \n",
       "boost_strength_factor          False                False         False   \n",
       "k_inference_factor             False                False         False   \n",
       "cnn_percent_on                 False                False         False   \n",
       "cnn_weight_sparsity            False                False         False   \n",
       "noise_accuracy_max             False                False         False   \n",
       "test_accuracy_max              False                False         False   \n",
       "\n",
       "                       momentum  batch_size  batches_in_epoch  boost_strength  \\\n",
       "learning_rate             False       False             False           False   \n",
       "learning_rate_gamma       False       False             False           False   \n",
       "weight_decay              False       False             False           False   \n",
       "momentum                   True       False             False           False   \n",
       "batch_size                False        True             False           False   \n",
       "batches_in_epoch          False       False              True           False   \n",
       "boost_strength            False       False             False            True   \n",
       "boost_strength_factor     False       False             False           False   \n",
       "k_inference_factor        False       False             False           False   \n",
       "cnn_percent_on            False       False             False           False   \n",
       "cnn_weight_sparsity       False       False             False           False   \n",
       "noise_accuracy_max        False       False             False           False   \n",
       "test_accuracy_max         False       False             False           False   \n",
       "\n",
       "                       boost_strength_factor  k_inference_factor  \\\n",
       "learning_rate                          False               False   \n",
       "learning_rate_gamma                    False               False   \n",
       "weight_decay                           False               False   \n",
       "momentum                               False               False   \n",
       "batch_size                             False               False   \n",
       "batches_in_epoch                       False               False   \n",
       "boost_strength                         False               False   \n",
       "boost_strength_factor                   True               False   \n",
       "k_inference_factor                     False                True   \n",
       "cnn_percent_on                         False               False   \n",
       "cnn_weight_sparsity                    False               False   \n",
       "noise_accuracy_max                     False               False   \n",
       "test_accuracy_max                      False               False   \n",
       "\n",
       "                       cnn_percent_on  cnn_weight_sparsity  \\\n",
       "learning_rate                   False                False   \n",
       "learning_rate_gamma             False                False   \n",
       "weight_decay                    False                False   \n",
       "momentum                        False                False   \n",
       "batch_size                      False                False   \n",
       "batches_in_epoch                False                False   \n",
       "boost_strength                  False                False   \n",
       "boost_strength_factor           False                False   \n",
       "k_inference_factor              False                False   \n",
       "cnn_percent_on                   True                False   \n",
       "cnn_weight_sparsity             False                 True   \n",
       "noise_accuracy_max               True                False   \n",
       "test_accuracy_max                True                False   \n",
       "\n",
       "                       noise_accuracy_max  test_accuracy_max  \n",
       "learning_rate                       False              False  \n",
       "learning_rate_gamma                 False              False  \n",
       "weight_decay                        False              False  \n",
       "momentum                            False              False  \n",
       "batch_size                          False              False  \n",
       "batches_in_epoch                    False              False  \n",
       "boost_strength                      False              False  \n",
       "boost_strength_factor               False              False  \n",
       "k_inference_factor                  False              False  \n",
       "cnn_percent_on                       True               True  \n",
       "cnn_weight_sparsity                 False              False  \n",
       "noise_accuracy_max                   True               True  \n",
       "test_accuracy_max                    True               True  "
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[corr_params].astype(np.float32).corr() > 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>learning_rate</th>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>momentum</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <th>boost_strength</th>\n",
       "      <th>boost_strength_factor</th>\n",
       "      <th>k_inference_factor</th>\n",
       "      <th>cnn_percent_on</th>\n",
       "      <th>cnn_weight_sparsity</th>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <th>test_accuracy_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>learning_rate</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>learning_rate_gamma</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weight_decay</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>momentum</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>batch_size</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>batches_in_epoch</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boost_strength</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>boost_strength_factor</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>k_inference_factor</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn_percent_on</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cnn_weight_sparsity</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>noise_accuracy_max</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>test_accuracy_max</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       learning_rate  learning_rate_gamma  weight_decay  \\\n",
       "learning_rate                  False                False         False   \n",
       "learning_rate_gamma            False                False         False   \n",
       "weight_decay                   False                False         False   \n",
       "momentum                       False                False         False   \n",
       "batch_size                     False                False         False   \n",
       "batches_in_epoch               False                False         False   \n",
       "boost_strength                 False                False         False   \n",
       "boost_strength_factor          False                False         False   \n",
       "k_inference_factor             False                False         False   \n",
       "cnn_percent_on                 False                False         False   \n",
       "cnn_weight_sparsity            False                False         False   \n",
       "noise_accuracy_max             False                False         False   \n",
       "test_accuracy_max              False                False         False   \n",
       "\n",
       "                       momentum  batch_size  batches_in_epoch  boost_strength  \\\n",
       "learning_rate             False       False             False           False   \n",
       "learning_rate_gamma       False       False             False           False   \n",
       "weight_decay              False       False             False           False   \n",
       "momentum                  False       False             False           False   \n",
       "batch_size                False       False             False           False   \n",
       "batches_in_epoch          False       False             False           False   \n",
       "boost_strength            False       False             False           False   \n",
       "boost_strength_factor     False       False             False           False   \n",
       "k_inference_factor        False       False             False           False   \n",
       "cnn_percent_on            False       False             False           False   \n",
       "cnn_weight_sparsity       False       False             False           False   \n",
       "noise_accuracy_max         True       False             False           False   \n",
       "test_accuracy_max         False       False             False           False   \n",
       "\n",
       "                       boost_strength_factor  k_inference_factor  \\\n",
       "learning_rate                          False               False   \n",
       "learning_rate_gamma                    False               False   \n",
       "weight_decay                           False               False   \n",
       "momentum                               False               False   \n",
       "batch_size                             False               False   \n",
       "batches_in_epoch                       False               False   \n",
       "boost_strength                         False               False   \n",
       "boost_strength_factor                  False               False   \n",
       "k_inference_factor                     False               False   \n",
       "cnn_percent_on                         False               False   \n",
       "cnn_weight_sparsity                    False               False   \n",
       "noise_accuracy_max                     False               False   \n",
       "test_accuracy_max                      False               False   \n",
       "\n",
       "                       cnn_percent_on  cnn_weight_sparsity  \\\n",
       "learning_rate                   False                False   \n",
       "learning_rate_gamma             False                False   \n",
       "weight_decay                    False                False   \n",
       "momentum                        False                False   \n",
       "batch_size                      False                False   \n",
       "batches_in_epoch                False                False   \n",
       "boost_strength                  False                False   \n",
       "boost_strength_factor           False                False   \n",
       "k_inference_factor              False                False   \n",
       "cnn_percent_on                  False                False   \n",
       "cnn_weight_sparsity             False                False   \n",
       "noise_accuracy_max              False                False   \n",
       "test_accuracy_max               False                False   \n",
       "\n",
       "                       noise_accuracy_max  test_accuracy_max  \n",
       "learning_rate                       False              False  \n",
       "learning_rate_gamma                 False              False  \n",
       "weight_decay                        False              False  \n",
       "momentum                             True              False  \n",
       "batch_size                          False              False  \n",
       "batches_in_epoch                    False              False  \n",
       "boost_strength                      False              False  \n",
       "boost_strength_factor               False              False  \n",
       "k_inference_factor                  False              False  \n",
       "cnn_percent_on                      False              False  \n",
       "cnn_weight_sparsity                 False              False  \n",
       "noise_accuracy_max                  False              False  \n",
       "test_accuracy_max                   False              False  "
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[corr_params].astype(np.float32).corr() < -0.3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Positive correlation: cnn_percent_on with noise_accuracy_max and test_accuracy_max\n",
    "- Negative correlation: momentum and noise_accuracy_max"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Further analysis on the tunable hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 234,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 237,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Noise accuracy\n",
      "[('learning_rate', -0.008320055931102738),\n",
      " ('learning_rate_gamma', 0.00010922925444718565),\n",
      " ('weight_decay', -0.007074033772320695),\n",
      " ('momentum', -0.023921331255530413),\n",
      " ('batch_size', 0.017012582555930997),\n",
      " ('batches_in_epoch', 0.0065225404456871194),\n",
      " ('boost_strength', 0.005106812347459978),\n",
      " ('boost_strength_factor', -0.00043002669351718305),\n",
      " ('k_inference_factor', -0.004576034582844502),\n",
      " ('cnn_percent_on', 0.008260245666037353),\n",
      " ('cnn_weight_sparsity', 0.0016398916756805678)]\n",
      "\n",
      " Test accuracy\n",
      "[('learning_rate', -0.0037434259712621066),\n",
      " ('learning_rate_gamma', 0.00355353048512084),\n",
      " ('weight_decay', -0.00425357050234218),\n",
      " ('momentum', -0.03087870199997384),\n",
      " ('batch_size', 0.04314875995376824),\n",
      " ('batches_in_epoch', 0.016936418027689627),\n",
      " ('boost_strength', 0.013619723602916817),\n",
      " ('boost_strength_factor', -0.0018614337515772711),\n",
      " ('k_inference_factor', -0.0158375244155911),\n",
      " ('cnn_percent_on', 0.045455169207160875),\n",
      " ('cnn_weight_sparsity', 0.0009522081649769887)]\n"
     ]
    }
   ],
   "source": [
    "# Including all experiments with 30 or more epochs\n",
    "df_inf = df[df['epochs']>=30]\n",
    "y1 = df_inf['noise_accuracy_max']\n",
    "y2 = df_inf['test_accuracy_max']\n",
    "X = df_inf[tunable_params].astype(np.float32)\n",
    "\n",
    "# adjust all X to same scale\n",
    "scaler = StandardScaler()\n",
    "X = scaler.fit_transform(X)\n",
    "\n",
    "model_noise = LinearRegression()\n",
    "model_noise.fit(X, y1)\n",
    "\n",
    "print(\"\\n Noise accuracy\")\n",
    "pprint(list(zip(tunable_params, model_noise.coef_)))\n",
    "\n",
    "model_test = LinearRegression()\n",
    "model_test.fit(X, y2)\n",
    "\n",
    "print(\"\\n Test accuracy\")\n",
    "pprint(list(zip(tunable_params, model_test.coef_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 238,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " Noise accuracy\n",
      "[('learning_rate', -0.013711040159944263),\n",
      " ('learning_rate_gamma', -0.0001959410878818779),\n",
      " ('weight_decay', -0.013152972568734067),\n",
      " ('momentum', -0.021176038112329668),\n",
      " ('batch_size', 0.010269630267931313),\n",
      " ('batches_in_epoch', -0.00014625127560307852),\n",
      " ('boost_strength', 0.0010431710438098365),\n",
      " ('boost_strength_factor', -0.00037476933729079377),\n",
      " ('k_inference_factor', -0.0008177660350967232),\n",
      " ('cnn_percent_on', -0.0006308944388026454),\n",
      " ('cnn_weight_sparsity', -0.001361569311950573)]\n",
      "\n",
      " Test accuracy\n",
      "[('learning_rate', 0.003507782817232606),\n",
      " ('learning_rate_gamma', -0.0014039065080307283),\n",
      " ('weight_decay', 0.000412071505479126),\n",
      " ('momentum', -0.007036921311942076),\n",
      " ('batch_size', 0.013137881191602896),\n",
      " ('batches_in_epoch', 0.0045387770677112515),\n",
      " ('boost_strength', 0.00018761581131504984),\n",
      " ('boost_strength_factor', -0.0024625983394766736),\n",
      " ('k_inference_factor', -0.002469787063475652),\n",
      " ('cnn_percent_on', 0.0201868856868994),\n",
      " ('cnn_weight_sparsity', 0.009009976498501736)]\n"
     ]
    }
   ],
   "source": [
    "# Including all experiments with 90 or more epochs (\"completed\")\n",
    "df_inf = df[df['epochs']>=90]\n",
    "y1 = df_inf['noise_accuracy_max']\n",
    "y2 = df_inf['test_accuracy_max']\n",
    "X = df_inf[tunable_params].astype(np.float32)\n",
    "\n",
    "# adjust all X to same scale\n",
    "scaler = StandardScaler()\n",
    "X = scaler.fit_transform(X)\n",
    "\n",
    "model_noise = LinearRegression()\n",
    "model_noise.fit(X, y1)\n",
    "\n",
    "print(\"\\n Noise accuracy\")\n",
    "pprint(list(zip(tunable_params, model_noise.coef_)))\n",
    "\n",
    "model_test = LinearRegression()\n",
    "model_test.fit(X, y2)\n",
    "\n",
    "print(\"\\n Test accuracy\")\n",
    "pprint(list(zip(tunable_params, model_test.coef_)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- As correlation already showed, cnn_percent_on and momentum seems to have the greatest impact. The first is expected, but momentum is actually an interesting finding, specially since it is negatively correlated in the sparse model - a smaller momentum would lead to a higher noise accuracy. Why is that? \n",
    "- cnn_percent_on impacts specially the test accuracy, indicating sparsity would actually have a negative impact on test performance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### What is the ideal value for each variable that maximizes both accuracies?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Only included complete experiments\n",
    "df_inf = df[df['epochs']>=90][corr_params]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {},
   "outputs": [],
   "source": [
    "def stats(arr):\n",
    "  return [round(v, 4) for v in [np.min(arr), np.mean(arr), np.max(arr)]]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 277,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate            [0.0628, 0.0985, 0.1241]\n",
       "learning_rate_gamma       [0.0511, 0.0907, 0.129]\n",
       "weight_decay              [0.0006, 0.0008, 0.001]\n",
       "momentum                 [0.3494, 0.5863, 0.8658]\n",
       "batch_size                       [64, 115.2, 128]\n",
       "batches_in_epoch                [399, 454.8, 529]\n",
       "boost_strength           [1.3055, 1.6327, 1.7856]\n",
       "boost_strength_factor    [0.5093, 0.6222, 0.8509]\n",
       "k_inference_factor       [0.8662, 1.0072, 1.0956]\n",
       "cnn_percent_on           [0.2615, 0.3124, 0.3481]\n",
       "cnn_weight_sparsity      [0.8783, 0.9242, 0.9664]\n",
       "dtype: object"
      ]
     },
     "execution_count": 277,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_inf.sort_values('test_accuracy_max', ascending=False)[tunable_params].head(5).apply(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 278,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate              [0.0532, 0.09, 0.1284]\n",
       "learning_rate_gamma      [0.0705, 0.1158, 0.1582]\n",
       "weight_decay             [0.0006, 0.0008, 0.0011]\n",
       "momentum                 [0.4962, 0.8295, 0.9323]\n",
       "batch_size                       [64, 102.4, 128]\n",
       "batches_in_epoch                [325, 412.4, 538]\n",
       "boost_strength            [0.948, 1.2459, 1.7351]\n",
       "boost_strength_factor    [0.5366, 0.8112, 0.9749]\n",
       "k_inference_factor       [0.9294, 1.0544, 1.1601]\n",
       "cnn_percent_on            [0.153, 0.2288, 0.3045]\n",
       "cnn_weight_sparsity      [0.2458, 0.4239, 0.6889]\n",
       "dtype: object"
      ]
     },
     "execution_count": 278,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_inf.sort_values('test_accuracy_max', ascending=True)[tunable_params].head(5).apply(stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Why is boost strength affecting the test accuracy? It does not have the same impact in noise accuracy. Hypothesis: it is a regularization for the amount of sparsity in the model. \n",
    "- Higher cnn_percent_on and cnn_weight_sparsity are indicatives of better test accuracy. Incidentally, they are also indicatives of better noise accuracy, which is unexpected\n",
    "- Lower weight decay improves noise accuracy, however has no impact on test accuracy. Weight decay would just make the network even more sparse in the cases where it is already too sparse, which can be a bigger evidence that too much sparsity is impacting performance\n",
    "- As expected, higher batch size and more batches per epoch improve both metrics\n",
    "- For noise accuracy, lower learning rate is preferred\n",
    "- Momentum ideal value seems to be between 0.5 and 0.6, and it has a high impact on the model. This is unexpected as usual values for SGD momentum in literature is around 0.9 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 279,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate            [0.0506, 0.0712, 0.1104]\n",
       "learning_rate_gamma       [0.053, 0.1039, 0.1594]\n",
       "weight_decay             [0.0003, 0.0006, 0.0011]\n",
       "momentum                 [0.3355, 0.4981, 0.6752]\n",
       "batch_size                       [64, 102.4, 128]\n",
       "batches_in_epoch                [325, 500.0, 585]\n",
       "boost_strength            [0.9152, 1.381, 1.7915]\n",
       "boost_strength_factor    [0.5029, 0.7229, 0.8583]\n",
       "k_inference_factor       [0.8895, 0.9761, 1.0884]\n",
       "cnn_percent_on           [0.2651, 0.2939, 0.3426]\n",
       "cnn_weight_sparsity      [0.4513, 0.8046, 0.9981]\n",
       "dtype: object"
      ]
     },
     "execution_count": 279,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_inf.sort_values('noise_accuracy_max', ascending=False)[tunable_params].head(5).apply(stats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 280,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "learning_rate            [0.0703, 0.1158, 0.1392]\n",
       "learning_rate_gamma      [0.0705, 0.1131, 0.1546]\n",
       "weight_decay              [0.0007, 0.001, 0.0011]\n",
       "momentum                 [0.4756, 0.7868, 0.9323]\n",
       "batch_size                        [64, 89.6, 128]\n",
       "batches_in_epoch                [338, 454.8, 545]\n",
       "boost_strength            [0.948, 1.3371, 1.7351]\n",
       "boost_strength_factor     [0.5046, 0.768, 0.9824]\n",
       "k_inference_factor       [0.8614, 1.0332, 1.1497]\n",
       "cnn_percent_on            [0.153, 0.2465, 0.3488]\n",
       "cnn_weight_sparsity      [0.2458, 0.4096, 0.6889]\n",
       "dtype: object"
      ]
     },
     "execution_count": 280,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_inf.sort_values('noise_accuracy_max', ascending=True)[tunable_params].head(5).apply(stats)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Supporting classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RayTuneExperimentBrowser(object):\n",
    "\n",
    "  \"\"\"\n",
    "  Class for browsing and manipulating experiment results directories created\n",
    "  by Ray Tune.\n",
    "  \"\"\"\n",
    "\n",
    "  def __init__(self, experiment_path):\n",
    "    self.experiment_path = os.path.abspath(experiment_path)\n",
    "    self.experiment_states = self._get_experiment_states(\n",
    "      self.experiment_path, exit_on_fail=True)\n",
    "\n",
    "    self.progress = {}\n",
    "    self.exp_directories = {}\n",
    "    self.checkpoint_directories = {}\n",
    "    self.params = {}\n",
    "    for experiment_state in self.experiment_states:\n",
    "      self._read_experiment(experiment_state)\n",
    "\n",
    "\n",
    "  def _read_experiment(self, experiment_state):\n",
    "    checkpoint_dicts = experiment_state[\"checkpoints\"]\n",
    "    checkpoint_dicts = [flatten_dict(g) for g in checkpoint_dicts]\n",
    "\n",
    "    for exp in checkpoint_dicts:\n",
    "      if exp.get(\"logdir\", None) is None:\n",
    "        continue\n",
    "      exp_dir = os.path.basename(exp[\"logdir\"])\n",
    "      csv = os.path.join(self.experiment_path, exp_dir, \"progress.csv\")\n",
    "      self.progress[exp[\"experiment_tag\"]] = pd.read_csv(csv)\n",
    "      self.exp_directories[exp[\"experiment_tag\"]] = os.path.abspath(\n",
    "        os.path.join(self.experiment_path, exp_dir))\n",
    "\n",
    "      # Figure out checkpoint file (.pt or .pth) if it exists. For some reason\n",
    "      # we need to switch to the directory in order for glob to work.\n",
    "      ed = os.path.abspath(os.path.join(self.experiment_path, exp_dir))\n",
    "      os.chdir(ed)\n",
    "      cds = glob.glob(\"checkpoint*\")\n",
    "      if len(cds) > 0:\n",
    "        cd = max(cds)\n",
    "        cf = glob.glob(os.path.join(cd, \"*.pt\"))\n",
    "        cf += glob.glob(os.path.join(cd, \"*.pth\"))\n",
    "        if len(cf) > 0:\n",
    "          self.checkpoint_directories[exp[\"experiment_tag\"]] = os.path.join(\n",
    "            ed, cf[0])\n",
    "        else:\n",
    "          self.checkpoint_directories[exp[\"experiment_tag\"]] = \"\"\n",
    "      else:\n",
    "        self.checkpoint_directories[exp[\"experiment_tag\"]] = \"\"\n",
    "\n",
    "      # Read in the configs for this experiment\n",
    "      paramsFile = os.path.join(self.experiment_path, exp_dir, \"params.json\")\n",
    "      with open(paramsFile) as f:\n",
    "        self.params[exp[\"experiment_tag\"]] = json.load(f)\n",
    "\n",
    "\n",
    "  def get_value(self, exp_substring=\"\",\n",
    "                tags=[\"test_accuracy\", \"noise_accuracy\"],\n",
    "                which='max'):\n",
    "    \"\"\"\n",
    "    For every experiment whose name matches exp_substring, scan the history\n",
    "    and return the appropriate value associated with tag.\n",
    "    'which' can be one of the following:\n",
    "        last: returns the last value\n",
    "         min: returns the minimum value\n",
    "         max: returns the maximum value\n",
    "      median: returns the median value\n",
    "    \n",
    "    Returns a pandas dataframe with two columns containing name and tag value\n",
    "    \"\"\"\n",
    "    # Collect experiment names that match exp at all\n",
    "    exps = [e for e in self.progress if exp_substring in e]\n",
    "\n",
    "    # empty histories always return None\n",
    "    columns = ['Experiment Name']\n",
    "    \n",
    "    # add the columns names for main tags\n",
    "    for tag in tags:\n",
    "      columns.append(tag)\n",
    "      columns.append(tag+'_'+which)\n",
    "      if which in [\"max\", \"min\"]:\n",
    "        columns.append(\"epoch_\"+str(tag))\n",
    "    \n",
    "    # add training iterations\n",
    "    columns.append('epochs')\n",
    "    \n",
    "    # add the remaining variables\n",
    "    columns.extend(self.params[exps[0]].keys())\n",
    "  \n",
    "    all_values = []\n",
    "    for e in exps:\n",
    "      # values for the experiment name\n",
    "      values = [e]\n",
    "      # values for the main tags\n",
    "      for tag in tags:\n",
    "        values.append(self.progress[e][tag].iloc[-1])\n",
    "        if which == \"max\":\n",
    "          values.append(self.progress[e][tag].max())\n",
    "          v = self.progress[e][tag].idxmax()\n",
    "          values.append(v)\n",
    "        elif which == \"min\":\n",
    "          values.append(self.progress[e][tag].min())\n",
    "          values.append(self.progress[e][tag].idxmin())\n",
    "        elif which == \"median\":\n",
    "          values.append(self.progress[e][tag].median())\n",
    "        elif which == \"last\":\n",
    "          values.append(self.progress[e][tag].iloc[-1])\n",
    "        else:\n",
    "          raise RuntimeError(\"Invalid value for which='{}'\".format(which))\n",
    "\n",
    "      # add number of epochs\n",
    "      values.append(self.progress[e]['training_iteration'].iloc[-1])\n",
    "          \n",
    "      # remaining values\n",
    "      for v in self.params[e].values():\n",
    "        if isinstance(v,list):\n",
    "          values.append(np.mean(v))\n",
    "        else:\n",
    "          values.append(v)         \n",
    "      \n",
    "      all_values.append(values)\n",
    "\n",
    "    p = pd.DataFrame(all_values, columns=columns)\n",
    "      \n",
    "    return p\n",
    "\n",
    "\n",
    "  def get_checkpoint_file(self, exp_substring=\"\"):\n",
    "    \"\"\"\n",
    "    For every experiment whose name matches exp_substring, return the\n",
    "    full path to the checkpoint file. Returns a list of paths.\n",
    "    \"\"\"\n",
    "    # Collect experiment names that match exp at all\n",
    "    exps = [e for e in self.progress if exp_substring in e]\n",
    "\n",
    "    paths = [self.checkpoint_directories[e] for e in exps]\n",
    "\n",
    "    return paths\n",
    "\n",
    "\n",
    "  def _get_experiment_states(self, experiment_path, exit_on_fail=False):\n",
    "    \"\"\"\n",
    "    Return every experiment state JSON file in the path as a list of dicts.\n",
    "    The list is sorted such that newer experiments appear later.\n",
    "    \"\"\"\n",
    "    experiment_path = os.path.expanduser(experiment_path)\n",
    "    experiment_state_paths = glob.glob(\n",
    "      os.path.join(experiment_path, \"experiment_state*.json\"))\n",
    "    if not experiment_state_paths:\n",
    "      if exit_on_fail:\n",
    "        print(\"No experiment state found!\")\n",
    "        sys.exit(0)\n",
    "      else:\n",
    "        return\n",
    "\n",
    "    experiment_state_paths = list(experiment_state_paths)\n",
    "    experiment_state_paths.sort()\n",
    "    experiment_states = []\n",
    "    for experiment_filename in list(experiment_state_paths):\n",
    "\n",
    "      with open(experiment_filename) as f:\n",
    "        experiment_states.append(json.load(f))\n",
    "\n",
    "    return experiment_states\n",
    "\n",
    "\n",
    "  def get_parameters(self, sorted_experiments):\n",
    "    for i,e in sorted_experiments.iterrows():\n",
    "      if e['Experiment Name'] in self.params:\n",
    "        params = self.params[e['Experiment Name']]\n",
    "        print(params['cnn_percent_on'][0])\n",
    "\n",
    "    print('test_accuracy')\n",
    "    for i,e in sorted_experiments.iterrows():\n",
    "      print(e['test_accuracy'])\n",
    "\n",
    "    print('noise_accuracy')\n",
    "    for i,e in sorted_experiments.iterrows():\n",
    "      print(e['noise_accuracy'])\n",
    "\n",
    "\n",
    "  def best_experiments(self, min_test_accuracy=0.86, min_noise_accuracy=0.785, sort_by=\"noise_accuracy\"):\n",
    "    \"\"\"\n",
    "    Return a dataframe containing all experiments whose best test_accuracy and\n",
    "    noise_accuracy are above the specified thresholds.\n",
    "    \"\"\"\n",
    "    best_accuracies = self.get_value()\n",
    "    best_accuracies.sort_values(sort_by, axis=0, ascending=False,\n",
    "                 inplace=True, na_position='last')\n",
    "    columns = best_accuracies.columns\n",
    "    best_experiments = pd.DataFrame(columns=columns)\n",
    "    for i, row in best_accuracies.iterrows():\n",
    "      if ((row[\"test_accuracy\"] > min_test_accuracy)\n",
    "           and (row[\"noise_accuracy\"] > min_noise_accuracy)):\n",
    "        best_experiments = best_experiments.append(row)\n",
    "\n",
    "    return best_experiments\n",
    "\n",
    "\n",
    "  def prune_checkpoints(self, max_test_accuracy=0.86, max_noise_accuracy=0.785):\n",
    "    \"\"\"\n",
    "    TODO: delete the checkpoints for all models whose best test_accuracy and\n",
    "    noise_accuracy are below the specified thresholds.\n",
    "    \"\"\"\n",
    "    pass\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
